import logging

import requests
import time
from bs4 import BeautifulSoup

'''
http://nj.sell.house365.com/community/cl_p563.html
    http://nj.sell.house365.com/community_id401672/
    http://nj.sell.house365.com/community_id401672/sell.html
'''


def do_house_list(plot_href, distinct_num, community_num):
    page_num = 1
    has_next_page = True
    while has_next_page:
        end_time = time.time()
        if page_num == 1:
            parse = request_http(plot_href + "/sell.html")
        else:
            parse = request_http(plot_href + '/sl_p' + str(page_num) + '.html')
        cost_time = str(time.time() - end_time)
        print(cost_time)
        logging.info("地区:%s,小区: %s,页数:%s,耗时:%s", distinct_num, community_num, page_num, str(cost_time))
        print("地区:{},小区:{},页数:{},耗时:{}".format(distinct_num, community_num, page_num, str(cost_time)))

        house_list = parse.find_all(class_='info')
        page_num = page_num + 1
        next_cricle = 1
        # parse.find_all(class_='pageItem')[parse.find_all(class_='pageItem').__len__() - 1].contents[0]['href'] #最后
        has_next_page = judge_has_next_page(parse)
        for house_one in house_list:
            # print(house_one)
            one_href = house_one.find(class_='name')['href']  # 'http://nj.sell.house365.com/s_180921880.html'
            one_name = house_one.find(class_='name').contents[0]  # 南师附小学区房蓁巷双南一北厨卫全明户型好无学籍占用急卖
            one_total_mony = house_one.find(class_='num').text  # 290
            one_total_unit = house_one.find(class_='priceIn').contents[1]  # 万元
            # one_unit = house_one.find(class_='unitPrice').contents[0].contents[0]  # 43939
            one_unit_mony = house_one.find(class_='number').text  # 43939
            one_unit_name = house_one.find(class_='unitPrice').contents[1]  # 元/㎡
            one_squre = house_one.find(class_='area').contents[0]  # 66.0㎡
            logging.info("%s %s %s %s %s %s %s", one_href, one_name,  one_total_mony,one_total_unit, one_unit_name, one_unit_mony,
                         one_squre)
            # print(one_href, one_name,  one_total_mony,one_total_unit,  one_unit_mony,one_unit_name, one_squre)
            next_cricle = next_cricle + 1
            if next_cricle > house_list.__len__() - 4:
                break


def judge_has_next_page(parse):
    return 'href' in parse.find_all(class_='pageItem')[parse.find_all(class_='pageItem').__len__() - 1].contents[
        0].attrs


sleep_second = 30


def request_http(plot_href):
    flag = True
    global sleep_second
    while flag:
        try:
            r = requests.get(plot_href)
            if r.status_code == 404:
                flag = True
            else:
                flag = False
                r.encoding = 'gbk'
                html = r.text
                parse = BeautifulSoup(html)
            if sleep_second > 0:
                sleep_second = sleep_second - 30
        except BaseException as ex:
            logging.info(ex)
            sleep_second = sleep_second + 30
            time.sleep(sleep_second)

    return parse


def do_distinct(distinct_href, distinct_num):
    parse = request_http(distinct_href)
    # print(parse)
    name = parse.find_all(class_='name')
    community_num = 1
    for e in name:
        plot_href = e['href']
        plot_name = e.find_all(class_='bold')[0].contents[0]
        # print(plot_name, plot_href)
        do_house_list(plot_href, distinct_num, community_num)
        community_num = community_num + 1

        # print(name)


time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
log_file = "./log/logger-" + str(time_now) + ".log"
logging.basicConfig(format='%(asctime)-15s %(message)s', handlers=[logging.FileHandler(log_file, 'w', 'utf-8')],
                    level=logging.INFO)

if __name__ == '__main__':
    distinct_num = 2
    while distinct_num <= 563:
        distinct_href = 'http://nj.sell.house365.com/community/cl_p' + str(distinct_num) + '.html'
        do_distinct(distinct_href, distinct_num)
        distinct_num = distinct_num + 1
        logging.info(distinct_num)
        # do_house_list('http://nj.sell.house365.com/community_id7892/sl_p27.html')
